package com.httpclient;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.http.Consts;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * httpclient 测试类 HttpClientTest.java
 * 
 * @author yaojinshan
 * @since 2018年5月21日下午5:23:54
 */
@SuppressWarnings({ "deprecation", "resource" })
public class HttpClientImageDownload {
	public static final String CHARSET_GBK = "GBK";
	static volatile int currentNum = 0;// 计数器
	static int threadCode = 0; 
	static class MultipleThread implements Runnable{
		private int start;
		private int end;
		public MultipleThread(int start,int end){
			this.start = start;
			this.end = end;
		}
		
		public int getStart() {
			return start;
		}

		public void setStart(int start) {
			this.start = start;
		}

		public int getEnd() {
			return end;
		}

		public void setEnd(int end) {
			this.end = end;
		}

		public void run() {
			Map<String, Map<Integer, String>> boxData = new HashMap<String, Map<Integer, String>>();
			try {
				String nextPageBody = "";
				for (int i = start ; i <= end; i++) {
					try {
						if(i == 1){
							nextPageBody = getRequest(domain + "/html/part/index17.html", CHARSET_GBK);
						}else{
							nextPageBody = getRequest(domain + "/html/part/index17_" + i + ".html", CHARSET_GBK);
						}
						getBoxBody(Jsoup.parse(nextPageBody), boxData);
					} catch (Exception e) {
						e.printStackTrace();
					}
				}
				// 循环获取详情页的数据
				for (String dateKey : boxData.keySet()) {
					Map<Integer, String> hrefMap = boxData.get(dateKey);
					File dir = new File("E:\\httpclient_data\\" + dateKey);
					if (!dir.exists()) {
						dir.mkdir();
					}
					try {
						downloadSource(hrefMap, dir);
					} catch (Exception e) {
						e.printStackTrace();
					}
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
			System.out.println(Thread.currentThread().getName() + " >>> 下载完成！");
		}
	}
	public static void main(String[] args) throws Exception {
		// 1.获取该类型的最大页码数
		String body = getRequest(domain + "/html/part/index17.html", CHARSET_GBK);
		if (null != body && !"".equals(body)) {
			//Document doc = Jsoup.parse(body);
			//int maxPageNum = getMaxPageNum(doc);//128
			int maxPageNum = 64;//128
			int pageSize = 8;//每页大小
			int pageNum = maxPageNum / pageSize;//总页数
			for (int pageIndex = 0; pageIndex < pageNum; pageIndex++) {
				if(pageIndex == 0){
					new Thread(new MultipleThread(1, pageSize * (pageIndex + 1)),"thread--" + pageIndex).start();
				}else if (pageIndex != 0 && pageIndex < (pageNum - 1)){
					new Thread(new MultipleThread(pageSize * pageIndex + 1, pageSize * (pageIndex + 1)),"thread--" + pageIndex).start();
				}else if(pageIndex == pageNum - 1){
					new Thread(new MultipleThread(pageSize * pageIndex + 1,maxPageNum),"thread--" + pageIndex).start();
				}
			}
			System.out.println("线程全部启动完毕，启动线程个数：" + pageNum);
			//System.out.println(boxData);
		}
		while(Thread.activeCount() > 0){
			Thread.yield();
		}
	}
	
	

	final static void downloadSource(Map<Integer, String> hrefMap, File dir) throws Exception {
		for (Integer iKey : hrefMap.keySet()) {
			String hrefStr = hrefMap.get(iKey);
			String detailBody = "";
			try {
				detailBody = getRequest(domain + hrefStr, CHARSET_GBK);
			} catch (Exception e) {
				e.printStackTrace();
			}
			if (null != detailBody && !"".equals(detailBody)) {
				Document detailDoc = Jsoup.parse(detailBody);
				String fileName = detailDoc.select(".page_title").get(0).text();
				File detailDir = new File(dir, fileName);
				// 创建文件目录
				if (!detailDir.exists()) {
					detailDir.mkdir();
					continue;
				}
				// 下载资源 获取content 第 0个元素
				Element content = detailDoc.select(".content").get(0);
				Elements imgElements = content.getElementsByTag("img");
				String imgUrl, imageName;
				for (Element imgEle : imgElements) {
					imgUrl = imgEle.attr("src");
					// imgSrc格式：https://xxx.freeimage.us/image.php?id=68FC_5A9003AA&jpg
					// imgSrc格式：http://www.s2tu.com/images/2018/02/12/IMG_1389.jpg
					//System.out.println(imgUrl);
					if (imgUrl.indexOf("image.php") != -1) {
						imageName = imgUrl.substring(imgUrl.lastIndexOf("=") + 1).replace("&", ".");
					} else {
						imageName = imgUrl.substring(imgUrl.lastIndexOf("/") + 1);
					}
					// System.out.println(imageName);
					downloadPicture(imgUrl, imageName, detailDir);
					// break;
				}
			}
		}
	}

	public static void downloadPicture(String imgUrl, String imageName, File detailDir) {
		try {
			// 构造URL
			URL url = new URL(imgUrl);
			// 打开连接
			URLConnection con = url.openConnection();
			con.setRequestProperty("User-Agent",
					"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
			// 设置请求超时为60s
			con.setConnectTimeout(30 * 1000);
			// 输入流
			InputStream is = con.getInputStream();

			// 1K的数据缓冲
			byte[] bs = new byte[1024];
			// 读取到的数据长度
			int len;
			// 输出的文件流
			File imgFile = new File(detailDir, imageName);
			if (!imgFile.exists()) {
				imgFile.createNewFile();
			}else{
				System.out.println(Thread.currentThread().getName() + " >>> downlaod repeat:" + imgUrl);
				return;
			}
			OutputStream os = new FileOutputStream(imgFile);
			// 开始读取
			while ((len = is.read(bs)) != -1) {
				os.write(bs, 0, len);
			}
			// 完毕，关闭所有链接
			os.close();
			is.close();
			System.out.println(Thread.currentThread().getName() + " >>> downlaod success:" + imgUrl);
		} catch (MalformedURLException e) {
			System.out.println(Thread.currentThread().getName() + " >>> downlaod fail:" + imgUrl + "----> MalformedURLException");
		} catch (FileNotFoundException e) {
			System.out.println(Thread.currentThread().getName() + " >>> downlaod fail:" + imgUrl + "----> FileNotFoundException");
		} catch (IOException e) {
			System.out.println(Thread.currentThread().getName() + " >>> downlaod fail:" + imgUrl + "----> IOException");
		}
	}

	public static int getMaxPageNum(Document doc) {
		Elements paginationElements = doc.select(".pagination");
		Element last = paginationElements.get(0).getElementsByTag("a").last();
		String href = last.attr("href");
		// 截取 字符串/html/part/index17_128.html
		return Integer.parseInt(href.substring(href.indexOf("_") + 1, href.indexOf(".")));
	}

	/**
	 * 数据存储格式为： 05-02 : <0自增,String> , 05-03 : <0自增,String>
	 * 
	 * @param boxData
	 */
	private static void getBoxBody(Document doc, Map<String, Map<Integer, String>> boxData) {
		// 获取 box list channel 节点
		Elements boxElement = doc.getElementsByClass("box list channel");
		// 获取第一个ul 节点
		Element ulElement = boxElement.get(0).getElementsByTag("ul").get(0);
		// System.out.println(ulElement);
		// 解析 ul 元素下边的 所有的 li节点的 a元素
		Elements select = ulElement.select("li a");

		String aHref = "";// a 标签的 href 属性
		Element dateElement_a = null;// a标签的 date元素
		// 遍历存放日期key 和 具体的href
		for (Element aElement : select) {
			// 格式 <a href="/html/article/index22440.html"
			// target="_blank"><span>05-02</span>text</a>
			dateElement_a = aElement.getElementsByTag("span").first();
			if (!boxData.keySet().contains(dateElement_a.text())) {
				boxData.put(dateElement_a.text(), null);
			}
			aHref = aElement.attr("href");
			dateElement_a = aElement.getElementsByTag("span").first();
			Map<Integer, String> tempData = boxData.get(dateElement_a.text());
			if (tempData == null) {
				tempData = new HashMap<Integer, String>();
			}
			currentNum++;
			tempData.put(currentNum, aHref);
			boxData.put(dateElement_a.text(), tempData);
		}
	}

	static String getRequest(String url, String charset){
		try {
			DefaultHttpClient httpclient = new DefaultHttpClient();
			HttpGet httpGet = new HttpGet(url);
			// HttpGet httpGet = new HttpGet("http://www.baidu.com");
			String body = "";
			HttpResponse response;
			HttpEntity entity;
			response = httpclient.execute(httpGet);
			entity = response.getEntity();
			body = EntityUtils.toString(entity, charset);// 这个就是页面源码了
			httpGet.abort();// 中断请求,接下来可以开始另一段请求
			// System.out.println("get: " + response.getStatusLine());// 这个可以打印状态
			// System.out.println(body);
			httpGet.releaseConnection();// 释放请求.如果释放了相当于要清空session
			int responseCode = response.getStatusLine().getStatusCode();
			if (Integer.valueOf(responseCode) < 400) {
				return body;
			}
		} catch (ClientProtocolException e) {
			System.out.println("request error : ---->ClientProtocolException---->" + url );
		}catch (IOException e) {
			System.out.println("request error : ---->IOException---->" + url );
		}
		return null;
	}

	static void postRequest() throws Exception {
		DefaultHttpClient httpclient = new DefaultHttpClient();
		String body = "";
		HttpResponse response;
		HttpEntity entity;
		HttpPost httpPost = new HttpPost("http://www.baidu.com");// 一定要改成可以提交的地址,这里用百度代替
		List<NameValuePair> nvps = new ArrayList<NameValuePair>();
		nvps.add(new BasicNameValuePair("name", "1"));// 名值对
		nvps.add(new BasicNameValuePair("account", "xxxx"));
		httpPost.setEntity(new UrlEncodedFormEntity(nvps, Consts.UTF_8));
		response = httpclient.execute(httpPost);
		entity = response.getEntity();
		body = EntityUtils.toString(entity, "utf-8");
		System.out.println("Login form get: " + response.getStatusLine());// 这个可以打印状态
		httpPost.abort();
		System.out.println(body);
		httpPost.releaseConnection();
	}

	static final String domain = "http://www.2011xx.com";
}
